#!/usr/bin/env python
# -*-coding:UTF-8 -*-
'''
@Project ：19-xpath站长素材.py
@File：42-异步爬虫百度小说.py
@Author ：文非
@Date：2021/3/1520:52
@Require：# 百度小说西游记的爬取
'''

# http://dushu.baidu.com/api/pc/getCatalog?data={"book_id":"4305713956"} 第一个url "代表引号 --- 所有章节的名称
# http://dushu.baidu.com/api/pc/getChapterContent?data={"book_id":"4305713956","cid":"4305713956|10507127","need_bookinfo":1}-#小说内容
import json

import aiofiles
import requests
import asyncio
import time
import aiohttp

async def download_novel(cid, book_id, title,sema):
    """获取小说内容"""
    # data
    data = {
        "book_id": book_id,
        "cid": f"{book_id}|{cid}",
        "need_bookinfo": 1
    }
    data = json.dumps(data)
    # 指定url
    url = f"http://dushu.baidu.com/api/pc/getChapterContent?data={data}"
    async with sema:
        async with aiohttp.ClientSession()as session:
            async with session.get(url)as res:
                content_json = await res.json()
                # print(content_json["data"]["novel"]["content"])
                # print(title,"开始下载")
                content=content_json["data"]["novel"]["content"]
                print(content)
                path="./元尊/"+title
                async with aiofiles.open(path, mode="w", encoding="utf-8")as fp:
                    await fp.write(content)
                    # print(title,"下载完成")

async def get_Cid(book_id,headers):
    """获取小说内容url里面的cid"""
    data = {
        "book_id": book_id
    }
    # 将json 转为string
    data = json.dumps(data)
    # 指定url
    url = f'http://dushu.baidu.com/api/pc/getCatalog?data={data}'
    # print(url)
    # 发送请求ajax获取动态加载js-json数据
    json_data = requests.get(url=url, headers=headers).json()
    cid_list = json_data["data"]["novel"]["items"]
    # print(cid_list)
    tasks = []
    sema = asyncio.Semaphore(100)
    for cid in cid_list:
        title = cid["title"]
        cid = cid["cid"]
        # 发起异步任务
        tasks.append(download_novel(cid, book_id, title,sema))

    await asyncio.wait(tasks)


def main():
    # UA伪装
    headers = {
        "User-Agent":"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36"
    }
    # 指定小说的book_id
    book_id = "4305713956"

    # 调用获取小说cid的函数
    asyncio.run(get_Cid(book_id, headers))
    """主程序入口"""

if __name__=="__main__":
    main()